import requests
import re
#from spider import Spider


class ReadNovelClass():
    
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
        }
    
    def __init__(self,
                 charset = 'utf-8',
                 keyword = None,
                 search_url = 'https://www.readnovel.com/search?kw={}&pageNum={}',
                 page_num = 1,
                 free_url = None,
                 book_url = None,
                 chapter_url = None,
                 author_url = None,
                 json = None,
                
                 ):
        
        self.search_url = search_url
        self.page_num = page_num
        self.charset = charset
        self.json = json
        self.s = requests.Session()
        #self.s.headers.update(self.headers)
        
        if keyword != None:
            self.keyword = keyword
            
            self.s_html = self.novel_search()
        if book_url != None:
            self.book_url = book_url
            
            self.book_h5 = self.book_html()
        if chapter_url != None:
            self.chapter_url = chapter_url
            self.detail_h5 = self.detail_html()
        if free_url != None:
            self.free_url = free_url
            self.free_h5 = self.free_html()
        if author_url != None:
            self.author_url = author_url
            self.author_h5 = self.author_html()

    def get_html(self,url,charset,headers = None,json = None): #传入url，字符编码，headers，返回源代码

        res = requests.get(url,headers = headers)
        res.encoding = charset

        if json != None:
            return res.json()
        return res.text

    def get_info(self,html_info,**regex):   #传入网页源代码，在里面查找符合正则表达式的结果并以字典返回
        
        info_dict = {}
        for key,value in regex.items():
            info_dict[key] = re.findall(value,html_info,re.S)

        return info_dict
    
    def novel_search(self,
                     #search_url = 'https://www.qidian.com/search?kw={}&page={}',
                     ):
        new_url = self.search_url.format(self.keyword,self.page_num)
        print(new_url)
        #res_text = self.get_html(url = new_url,charset = self.charset,headers = self.headers)
        #return res_text
        
        res = requests.get(new_url,headers = self.headers)
        res.encoding = self.charset
        if self.json != None:
            return res.json
        #print(res.text)
        return res.text
    def search_page(self,
                    max_page = '<div class="pagination fr" id="page-container" data-page="1" data-pageMax="(.*?)"></div>',
                    ):
        s_html = self.s_html
        info_dict = self.get_info(html_info = s_html,
                                  max_page = max_page,)
        print(info_dict)
        return info_dict
        
    def search_info(self,
                    novel_name = '<div class="book-mid-info">.*?<h4><a href=".*?" target="_blank">(.*?)</a></h4>',
                    novel_url = '<div class="book-mid-info">.*?<h4><a href="//(.*?)" target="_blank">.*?</a></h4>',
                    #author = '<a class="name" data-eid="qd_S06" href=".*?" target="_blank">(.*?)</a>',
                    novel_author = '<a class="name default">(.*?)</a>',
                    novel_status = '<em>\|</em><span>(.*?)</span>',
                    novel_intro = '<p class="intro">(.*?)</p>',
                    novel_img = '<div class="book-img-box">.*?<a href=".*?" target="_blank"><img src="//(.*?)"></a>',
                    latest_chapter = '<p class="update"><a href=".*?" target="_blank">(.*?)</a>',
                    latest_chapter_url = '<p class="update"><a href="//(.*?)" target="_blank">.*?</a>',
                    refresh_time = '<em>·</em><span>(.*?)</span>',
                    novel_type = '</em><a href=".*?" target="_blank">(.*?)</a>',
                    ):
        #小说URL：//book.qidian.com/info/
        #作者URL：//my.qidian.com/author/
        s_html = self.s_html
        info_dict = self.get_info(html_info = s_html,
                                  novel_name = novel_name,novel_url = novel_url,
                                  novel_author = novel_author,
                                  novel_status = novel_status,novel_intro = novel_intro,
                                  latest_chapter = latest_chapter,latest_chapter_url = latest_chapter_url,
                                  refresh_time = refresh_time,novel_img = novel_img,novel_type = novel_type,
                                  )
        #print(info_dict)
        '''
        url_list = []
        for eve_url in info_dict['latest_chapter_url']:
            if eve_url.startswith('//read'):
                url_list.append(eve_url)
            else:
                url_list.append('')
        info_dict['latest_chapter_url'] = url_list
        '''
        '''
        if len(info_dict['novel_author_url']) < len(info_dict['novel_name']):
            for i in range(len(info_dict['novel_name']) - len(info_dict['novel_author_url'])):
                info_dict['novel_author_url'].append('')
        '''
        novel_dict = {}
        for i in range(len(info_dict['novel_name'])):
            novel_dict[info_dict['novel_name'][i]\
                       .replace('<cite class="red-kw">','')\
                       .replace('</cite>','')] = {'novel_url':info_dict['novel_url'][i],
                                                  'novel_author':info_dict['novel_author'][i],
                                                  'novel_status':info_dict['novel_status'][i],
                                                  'novel_intro':info_dict['novel_intro'][i],
                                                  'latest_chapter':info_dict['latest_chapter'][i],
                                                  'refresh_time':info_dict['refresh_time'][i],
                                                  'latest_chapter_url':info_dict['latest_chapter_url'][i],
                                                  'novel_img':info_dict['novel_img'][i],
                                                  'novel_type':info_dict['novel_type'][i],
                                                  }
        
        #info_dict['novel_dict'] = novel_dict
        #print(novel_dict)
        return novel_dict
        
    def book_html(self):

        book_url = 'https://www.readnovel.com/book/{}'    #这里是拼接链接的链接
        
        res = requests.get(self.book_url,headers = self.headers)
        res.encoding = self.charset
        if self.json != None:
            return res.json
        return res.text
    def book_info(self,
                  author_name = '<a class="writer default" target="_blank">(.*?)著</a></h1>',
                  book_intro = '<p class="intro">(.*?)</p>',
                  book_status = '<span class="tag">.*?<i class="blue">(.*?)</i>',
                  book_name = '<h1><em>(.*?)</em>',
                  refresh_time = '',
                  book_words = '<p class="total">.*?<span>(.*?)</span><em>(.*?)</em>',
                  book_type ='<i class="org">.*?</i>.*?<i>(.*?)</i>',
                  book_img = '<a class="J-getJumpUrl default" id="bookImg" data-bid=".*?" data-firstchapterjumpurl=""><img src="(.*?)"></a>',
                  free_num = '<i>&#183;</i>共(.*?)章<span class="free">免费</span>',
                  latest_chapter = '<div class="update"><p><a class="blue" href=".*?" target="_blank">(.*?)</a>',
                  latest_chapter_url = '<div class="update"><p><a class="blue" href="(.*?)" target="_blank">.*?</a>', 
                  ):
        book_html = self.book_h5
        info_dict = self.get_info(html_info = book_html,
                                  novel_author = author_name,
                                  novel_intro = book_intro,novel_status = book_status,
                                  novel_name = book_name,novel_words = book_words,
                                  novel_img = book_img,free_num = free_num,
                                  novel_type = book_type,latest_chapter = latest_chapter,
                                  latest_chapter_url = latest_chapter_url,
                                  )
        
        for key,value in info_dict.items():
            #print(value)
            #if len(value) != 0:
            if key != 'free_num' and key != 'novel_words' and key != 'latest_chapter_url':
                info_dict[key] = value[0]
        info_dict['latest_chapter_url'] = 'www.readnovel.com' + info_dict['latest_chapter_url'][0]
        #if info_dict['latest_chapter_url'][0].startswith('//vip'):
            #info_dict['latest_chapter_url'] = None
        info_dict['novel_author_url'] = None
        info_dict['novel_words'] = info_dict['novel_words'][0][0] + info_dict['novel_words'][0][1]
        free_num_list = info_dict['free_num']
        free_num = 0
        for i in free_num_list:
            free_num += int(i)
        info_dict['free_num'] = free_num
        print(free_num)
        print(type(free_num))
        #print(info_dict)
        return info_dict
        
    def chapter_info(self,
                     chapter_name = '<li data-rid=".*?"><a href=".*?" target="_blank" data-cid=".*?" title=".*?">(.*?)</a>',
                     chapter_url = '<li data-rid=".*?"><a href="//(.*?)" target="_blank" data-cid=".*?" title=".*?">.*?</a>',
                     chapter_info = '<li data-rid=".*?"><a href=".*?" target="_blank" data-cid=".*?" title="(.*?)">.*?</a>',
                     free_num = '<i>&#183;</i>共(.*?)章<span class="free">免费</span>',
                     ):
        book_html = self.book_h5
        info_dict = self.get_info(html_info = book_html,
                                  chapter_name = chapter_name,chapter_url = chapter_url,
                                  chapter_info = chapter_info,free_num = free_num,
                                  )
        #print(info_dict)
        free_num_list = info_dict['free_num']
        free_num = 0
        for i in free_num_list:
            free_num += int(i)
        #free_num = int(free_num)
        chapter_dict = {}
        for i in range(free_num):
            chapter_dict[info_dict['chapter_name'][i]] = [info_dict['chapter_url'][i],
                                                          info_dict['chapter_info'][i],
                                                          ]
        #print(chapter_dict)
        return chapter_dict
    def detail_html(self):
        chapter_url = 'https://www.readnovel.com/chapter/{}'  #这里是拼接链接的链接
        
        res = requests.get(self.chapter_url,headers = self.headers)
        res.encoding = self.charset
        if self.json != None:
            return res.json
        return res.text
    
    def detail_info(self,
                    detail_title = '<h3 class="j_chapterName">(.*?)</h3>',
                    detail_book = '<div class="info fl">\s*?<a href=".*?" target="_blank" data-bid=".*?" data-auid=".*?"><em class="iconfont">&#xe60c;</em>(.*?)</a>',
                    detail_bookurl = '<div class="info fl">\s*?<a href="(.*?)" target="_blank" data-bid=".*?" data-auid=".*?"><em class="iconfont">&#xe60c;</em>.*?</a>',
                    #detail_auurl = '<a href="//me.qidian.com/authorIndex.aspx\?id\=(\d*)" target="_blank" data-bid=".*?" data-auid=".*?" data-eid="qd_R103"><em class="iconfont">&#xe650;</em>.*?</a>',
                    detail_author = '<em class="iconfont">&#xe650;</em>(.*?)</a>',
                    detail_fenlei = '<a href=".*?" target="_blank" data-chanid=".*?" id="j_chanId">(.*?)</a><em class="iconfont">&#xe621;</em>',
                    detail_update = '<i><em class="iconfont">&#xe653;</em><span class="j_updateTime">(.*?)</span></i>',
                    #detail_status = '<li><p><em>(.*?)</em>万</p><span>(.*?)（字）</span></li>', 
                    detail_word = '<i><em class="iconfont">&#xe64f;</em><span class="j_chapterWordCut">(.*?)</span>字</i>',
                    detail_time = '<i><em class="iconfont">&#xe653;</em><span class="j_updateTime">(.*?)</span></i>',
                    detail_content = '<div class="read-content j_readContent">(.*?)</div>',
                    detail_pre = '<a id="j_chapterPrev"   href="//(.*?)">上一章</a><span>\|</span>',
                    detail_cat = '</span>.*?<a href="//(.*?)" target="_blank">目录</a><span>|</span>',
                    #<a id="j_chapterPrev"   href=".*?">上一章</a><span>|</span>.*?<a href="//(.*?)" target="_blank">\
                    #目录</a><span>|</span>
                    detail_next = '<a id="j_chapterNext" href="//(.*?)">下一章</a>',
                    ):
        detail_h5 = self.detail_h5
        info_dict = self.get_info(html_info = detail_h5,
                                  chapter_name = detail_title,novel_author = detail_author,
                                  novel_name = detail_book,novel_url = detail_bookurl,
                                  novel_type = detail_fenlei,
                                  refresh_time = detail_update,
                                  novel_words = detail_word,detail_time = detail_time,
                                  chapter_content = detail_content,pre_chapter = detail_pre,
                                  novel_catalog = detail_cat,next_chapter = detail_next,
                                  )
        
        for key,value in info_dict.items():
            if len(value) != 0:
                info_dict[key] = value[0]
        #info_dict['detail_content'] = [info_dict['detail_content'][0].replace('<p>','')]
        info_dict['novel_status'] = None
        info_dict['novel_url'] = 'www.readnovel.com' + info_dict['novel_url']
        #print(info_dict)
        return info_dict

    def free_html(self):
        
        res = requests.get(self.free_url,headers = self.headers)
        res.encoding = self.charset
        if self.json != None:
            return res.json
        return res.text
    def free_page(self,
                  max_page = '<div class="pagination" id="page-container" data-total="(.*?)" data-size=".*?" data-page=".*?"',
                  ):
        free_html = self.free_h5
        
        info_dict = self.get_info(html_info = free_html,
                                  max_page = max_page,
                                  )
        print(info_dict)
        return info_dict
        
    def free_novel(self,
                   free_name = '<h3><a href=".*?" target="_blank" title=".*?">(.*?)</a></h3>',
                   free_url = '<h3><a href="(.*?)" target="_blank" title=".*?">.*?</a></h3>',
                   free_img = '<div class="book-img">.*?<a href=".*?" target="_blank" title=".*?">.*?<img src="//(.*?)" alt=".*?">',
                   free_author = '<h4><a class="default">(.*?)</a></h4>',
                   #free_auurl = '<p class="author">.*?<a class="name" href="(.*?)" .*?>.*?</a>',
                   free_fenlei = '<p class="tag"><span class="org">(.*?)</span><span class="red">.*?</span><span class="blue">.*?</span></p>',
                   free_status = '<p class="tag"><span class="org">.*?</span><span class="red">(.*?)</span><span class="blue">.*?</span></p>',
                   free_words = '<p class="tag"><span class="org">.*?</span><span class="red">.*?</span><span class="blue">(.*?)</span></p>',
                   free_intro = '<p class="intro">(.*?)</p>',
                   ):
        free_html = self.free_h5
        
        info_dict = self.get_info(html_info = free_html,
                                  free_name = free_name,free_url = free_url,
                                  free_author = free_author,free_words = free_words,
                                  free_fenlei = free_fenlei,free_status = free_status,
                                  free_intro = free_intro,free_img = free_img,
                                  )
        #print(info_dict)
        free_dict = {}
        for i in range(len(info_dict['free_name'])):
            free_dict[info_dict['free_name'][i]] = {'novel_url':'www.readnovel.com'+info_dict['free_url'][i],
                                                    'novel_author':info_dict['free_author'][i],
                                                    'novel_type':info_dict['free_fenlei'][i],
                                                    'novel_stauts':info_dict['free_status'][i],
                                                    'novel_intro':info_dict['free_intro'][i],
                                                    'novel_img':info_dict['free_img'][i],
                                                    'refresh_time':None,
                                                    'novel_words':info_dict['free_words'][i],
                                                    'latest_chapter':None,
                                                    'latest_chapter_url':None,
                                                    }
        #print(free_dict)
        return free_dict

    def author_html(self):

        author_url="https://my.qidian.com/author/{}"   #这里是作者拼接链接
        
        res = requests.get(self.author_url,headers = self.headers)
        res.encoding = self.charset
        if self.json != None:
            return res.json
        return res.text
    def author_message(self,
                       author_toux = '<img class="header-avatar-img" src="(.*?)" alt=".*?">',
                       author_jibie = '<span class="header-avatar-vip author-darkblue">(.*?)</span>',
                       author_name = '<h3>(.*?)<a class="header-msg-level icon icon-god-\d" href=".*?" title="level\d" target="_blank">.*?</a></h3>',
                       author_detail = '<div class="header-msg-desc">(.*?)</div>',
                       author_produ = '<span class="mr12">作品总数<strong class="header-msg-strong">(.*?)</strong></span><span class="ml12 mr12">累计字数<strong class="header-msg-strong">(.*?)\</strong>万</span><span class="ml12 mr12">创作天数<strong class="header-msg-strong">(.*?)</strong>天</div></span>',
                       author_protot = '<h2 class="author-title">全部作品(.*?)</h2>',
                       ):
        author_html = self.author_h5
        info_dict = self.get_info(html_info = author_html,
                                  author_toux = author_toux,author_jibie = author_jibie,
                                  author_name = author_name,author_detail = author_detail,
                                  author_produ = author_produ,author_protot = author_protot,
                                  )
        return info_dict
                       
    def author_info(self,
                    novel_name = '<div class="author-item-title"><a href=".*?" target="_blank" data-eid="qd_M269" data-bid=".*?">(.*?)</a></div>',
                    novel_url = '<div class="author-item-title"><a href="//book.qidian.com/info/(.*?)" target="_blank" data-eid="qd_M269" data-bid=".*?">.*?</a></div>',
                    novel_fenlei = '<div class="author-item-exp"><a href=".*?" target="_blank">(.*?)</a>',
                    novel_status = '<div class="author-item-exp"><a href=".*?" target="_blank">.*?</a><i class="author-item-line"></i><a href=".*?" target="_blank">(.*?)</a>',
                    novel_word = '<div class="author-item-exp"><a href=".*?" target="_blank">.*?</a><i class="author-item-line"></i><a href=".*?" target="_blank">.*?</a><i class="author-item-line"></i>(.*?)</div>',
                    novel_detail = '<div class="author-item-content">(.*?)</div>',
                    novel_update = '<div class="author-item-update"><a href=".*?" class="blue" target="_blank" title=".*?">(.*?)</a><span>(.*?)</span></div>',
                    novel_zxurl = '<div class="author-item-update"><a href="(.*?)" class="blue" target="_blank" title=".*?">.*?</a><span>.*?</span></div>',
                    novel_img = 'data-eid="qd_M268"><img src="(.*?)" srcset',
                    ):
        author_html = self.author_h5
        info_dict = self.get_info(html_info = author_html,
                                  novel_name = novel_name,novel_url = novel_url,
                                  novel_fenlei = novel_fenlei,novel_status = novel_status,
                                  novel_word = novel_word,novel_detail = novel_detail,
                                  novel_update = novel_update,novel_zxurl = novel_zxurl,
                                  novel_img = novel_img,
                                  )
        #print(info_dict)
        zxurl_list = []
        for eve_url in info_dict['novel_zxurl']:
            if eve_url.startswith('//read'):
                zxurl_list.append(eve_url.replace('//read.qidian.com/chapter/',''))
            else:
                zxurl_list.append('')
        info_dict['novel_zxurl'] = zxurl_list
        author_dict = {}
        for x in range(len(info_dict['novel_name'])):
            author_dict[info_dict['novel_name'][x]] = {'novel_url':info_dict['novel_url'][x],
                                                       'novel_fenlei':info_dict['novel_fenlei'][x],
                                                       'novel_status':info_dict['novel_status'][x],
                                                       'novel_word':info_dict['novel_word'][x],
                                                       'novel_detail':info_dict['novel_detail'][x],
                                                       'novel_update':info_dict['novel_update'][x],
                                                       'novel_zxurl':info_dict['novel_zxurl'][x],
                                                       'novel_img':info_dict['novel_img'][x],
                                                       }
        #print(author_dict)

        return author_dict
        
if __name__ == '__main__':
    book_url = 'https://www.readnovel.com/book/22302167000853302'
    chapter_url = 'https://www.readnovel.com/chapter/22302167000853302/95787142321813435'
    free_url = 'https://www.readnovel.com/free/all?pageSize=10&gender=2&catId=-1&isFinish=-1&isVip=1&size=-1&updT=-1&orderBy=0&pageNum={}'.format(1)
    author_url = 'https://my.qidian.com/author/1780247'
    keyword = '大主宰'
    a = ReadNovelClass(free_url = free_url)
    a.free_novel()
    #a = QdNovelClass(author_url = author_url)
    #print(a.author_message())
    #print('*'*100)
    #print(a.author_info())
    #print(a.chapter_info())
    #print(a.free_h5)
    #a.author_info()
